{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "# !wget https://f000.backblazeb2.com/file/malay-dataset/summarization/dailymail/translated-dailymail-train.json\n",
    "# !wget https://f000.backblazeb2.com/file/malay-dataset/summarization/cnn-news/translated-cnn-train.json"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "\n",
    "os.environ['CUDA_VISIBLE_DEVICES'] = ''\n",
    "os.environ['GOOGLE_APPLICATION_CREDENTIALS'] = 'mesolitica-tpu.json'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From /home/husein/t5/prepare/tokenization.py:135: The name tf.gfile.GFile is deprecated. Please use tf.io.gfile.GFile instead.\n",
      "\n"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "from numpy.random import default_rng\n",
    "import random\n",
    "import collections\n",
    "import re\n",
    "import tensorflow as tf\n",
    "from tqdm import tqdm\n",
    "import tokenization\n",
    "import json\n",
    "from google.cloud import storage\n",
    "import mp\n",
    "\n",
    "max_seq_length_encoder = 512\n",
    "max_seq_length_decoder = 512\n",
    "\n",
    "EOS_ID = 1\n",
    "\n",
    "tokenizer = tokenization.FullTokenizer(\n",
    "    vocab_file='pegasus.wordpiece', do_lower_case=False\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "def create_int_feature(values):\n",
    "    feature = tf.train.Feature(\n",
    "        int64_list=tf.train.Int64List(value=list(values))\n",
    "    )\n",
    "    return feature\n",
    "\n",
    "def write_instance_to_example_file(X, Y, output_file):\n",
    "    writer = tf.python_io.TFRecordWriter(output_file)\n",
    "    for i in range(len(X)):\n",
    "        input_ids = X[i]\n",
    "        target_ids = Y[i]\n",
    "        while len(input_ids) < max_seq_length_encoder:\n",
    "            input_ids.append(0)\n",
    "        while len(target_ids) < max_seq_length_decoder:\n",
    "            target_ids.append(0)\n",
    "\n",
    "        features = collections.OrderedDict()\n",
    "        features['input_ids'] = create_int_feature(input_ids)\n",
    "        features['target_ids'] = create_int_feature(target_ids)\n",
    "        tf_example = tf.train.Example(\n",
    "            features=tf.train.Features(feature=features)\n",
    "        )\n",
    "        writer.write(tf_example.SerializeToString())\n",
    "\n",
    "    tf.logging.info('Wrote %d total instances', inst_index)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_feature(x, y):\n",
    "    tokens = tokenizer.tokenize(x)\n",
    "    if len(tokens) > (max_seq_length_encoder - 2):\n",
    "        tokens = tokens[: max_seq_length_encoder - 2]\n",
    "    tokens = tokens\n",
    "\n",
    "    tokens_y = []\n",
    "    for y_ in y:\n",
    "        tokens_y.extend(tokenizer.tokenize(y_))\n",
    "    if len(tokens_y) > (max_seq_length_decoder - 1):\n",
    "        tokens_y = tokens_y[: max_seq_length_decoder - 1]\n",
    "    \n",
    "    tokens_y = tokenizer.convert_tokens_to_ids(tokens_y)\n",
    "    tokens_y = tokens_y + [EOS_ID]\n",
    "    return tokenizer.convert_tokens_to_ids(tokens), tokens_y"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "def write_instance_to_example_file(X, Y, output_file):\n",
    "    writer = tf.python_io.TFRecordWriter(output_file)\n",
    "    for i in range(len(X)):\n",
    "        input_ids = X[i]\n",
    "        target_ids = Y[i]\n",
    "        while len(input_ids) < max_seq_length_encoder:\n",
    "            input_ids.append(0)\n",
    "        while len(target_ids) < max_seq_length_decoder:\n",
    "            target_ids.append(0)\n",
    "\n",
    "        features = collections.OrderedDict()\n",
    "        features['input_ids'] = create_int_feature(input_ids)\n",
    "        features['target_ids'] = create_int_feature(target_ids)\n",
    "        tf_example = tf.train.Example(\n",
    "            features=tf.train.Features(feature=features)\n",
    "        )\n",
    "        writer.write(tf_example.SerializeToString())\n",
    "\n",
    "    tf.logging.info('Wrote %d total instances', len(X))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "def loop(rows):\n",
    "    client = storage.Client()\n",
    "    bucket = client.bucket('mesolitica-tpu-general')\n",
    "    rows, index = rows\n",
    "    \n",
    "    X, Y = [], []\n",
    "    output_file = f'pegasus-summarization-{index}.tfrecord'\n",
    "    for i in tqdm(range(len(rows))):\n",
    "        tokens, tokens_y = get_feature(' '.join(data[i]['ms_article']), [' '.join(data[i]['ms_abstract'])])\n",
    "        X.append(tokens)\n",
    "        Y.append(tokens_y)\n",
    "            \n",
    "    write_instance_to_example_file(X, Y, output_file)\n",
    "    blob = bucket.blob(f'pegasus-summarization-data/{output_file}')\n",
    "    blob.upload_from_filename(output_file)\n",
    "    os.system(f'rm {output_file}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('translated-cnn-train.json') as fopen:\n",
    "    data = json.load(fopen)\n",
    "    \n",
    "with open('translated-cnn-train.json') as fopen:\n",
    "    data.extend(json.load(fopen))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "import random\n",
    "\n",
    "random.shuffle(data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 9057/9057 [02:34<00:00, 58.53it/s]\n",
      "100%|██████████| 9057/9057 [02:33<00:00, 59.07it/s]\n",
      "100%|██████████| 9057/9057 [02:32<00:00, 59.25it/s]\n",
      "100%|██████████| 9057/9057 [02:34<00:00, 58.67it/s]\n",
      " 94%|█████████▍| 8506/9057 [02:28<00:08, 62.66it/s]\n",
      "100%|██████████| 9057/9057 [02:33<00:00, 59.13it/s]\n",
      " 97%|█████████▋| 8820/9057 [02:31<00:03, 62.34it/s]\n",
      "100%|██████████| 9057/9057 [02:34<00:00, 58.61it/s]\n",
      " 95%|█████████▌| 8631/9057 [02:28<00:08, 51.81it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Wrote 9057 total instances\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 96%|█████████▌| 8661/9057 [02:28<00:09, 43.56it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Wrote 9057 total instances\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 96%|█████████▌| 8717/9057 [02:32<00:05, 60.31it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Wrote 9057 total instances\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 96%|█████████▌| 8669/9057 [02:29<00:09, 42.48it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Wrote 9057 total instances\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████▉| 9053/9057 [02:33<00:00, 69.55it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Wrote 9057 total instances\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 9057/9057 [02:33<00:00, 58.98it/s]\n",
      "100%|██████████| 9057/9057 [02:35<00:00, 58.25it/s]\n",
      "100%|██████████| 9057/9057 [02:34<00:00, 58.56it/s]\n",
      "100%|██████████| 9057/9057 [02:35<00:00, 58.21it/s]\n",
      " 98%|█████████▊| 8904/9057 [02:32<00:01, 90.06it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Wrote 9057 total instances\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 97%|█████████▋| 8785/9057 [02:31<00:03, 72.70it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Wrote 9057 total instances\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 18/18 [00:00<00:00, 82.63it/s]t/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Wrote 18 total instances\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 98%|█████████▊| 8898/9057 [02:32<00:01, 88.32it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Wrote 9057 total instances\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 9057/9057 [02:34<00:00, 58.46it/s]]\n",
      "100%|██████████| 9057/9057 [02:34<00:00, 58.65it/s]\n",
      "100%|██████████| 9057/9057 [02:34<00:00, 58.78it/s] \n",
      "100%|██████████| 9057/9057 [02:36<00:00, 58.03it/s]\n",
      " 99%|█████████▉| 8978/9057 [02:33<00:00, 87.53it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Wrote 9057 total instances\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████▉| 9017/9057 [02:34<00:00, 81.31it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Wrote 9057 total instances\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 9057/9057 [02:37<00:00, 57.58it/s]\n",
      "100%|█████████▉| 9026/9057 [02:34<00:00, 78.19it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Wrote 9057 total instances\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 9057/9057 [02:34<00:00, 58.81it/s]\n",
      "100%|██████████| 9057/9057 [02:34<00:00, 58.46it/s]\n",
      "100%|██████████| 9057/9057 [02:34<00:00, 58.56it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Wrote 9057 total instances\n",
      "INFO:tensorflow:Wrote 9057 total instances\n",
      "INFO:tensorflow:Wrote 9057 total instances\n",
      "INFO:tensorflow:Wrote 9057 total instances\n",
      "INFO:tensorflow:Wrote 9057 total instances\n",
      "INFO:tensorflow:Wrote 9057 total instances\n",
      "INFO:tensorflow:Wrote 9057 total instances\n",
      "INFO:tensorflow:Wrote 9057 total instances\n",
      "INFO:tensorflow:Wrote 9057 total instances\n"
     ]
    }
   ],
   "source": [
    "mp.multiprocessing(data, loop, cores = 20)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "def _decode_record(record, name_to_features):\n",
    "    \"\"\"Decodes a record to a TensorFlow example.\"\"\"\n",
    "    example = tf.parse_single_example(record, name_to_features)\n",
    "\n",
    "    # tf.Example only supports tf.int64, but the TPU only supports tf.int32.\n",
    "    # So cast all int64 to int32.\n",
    "    for name in list(example.keys()):\n",
    "        t = example[name]\n",
    "        if t.dtype == tf.int64:\n",
    "            t = tf.to_int32(t)\n",
    "        example[name] = t\n",
    "\n",
    "    return example\n",
    "\n",
    "def input_fn_builder(\n",
    "    input_files,\n",
    "    max_seq_length_encoder,\n",
    "    max_seq_length_decoder,\n",
    "    max_predictions_per_seq,\n",
    "    is_training,\n",
    "    num_cpu_threads = 4,\n",
    "):\n",
    "    def input_fn(params):\n",
    "        batch_size = params['batch_size']\n",
    "\n",
    "        name_to_features = {\n",
    "            'input_ids': tf.FixedLenFeature([max_seq_length_encoder], tf.int64),\n",
    "            'target_ids': tf.FixedLenFeature(\n",
    "                [max_seq_length_decoder], tf.int64\n",
    "            ),\n",
    "        }\n",
    "        if is_training:\n",
    "            d = tf.data.Dataset.from_tensor_slices(tf.constant(input_files))\n",
    "            d = d.repeat()\n",
    "            d = d.shuffle(buffer_size = len(input_files))\n",
    "            cycle_length = min(num_cpu_threads, len(input_files))\n",
    "            d = d.apply(\n",
    "                tf.contrib.data.parallel_interleave(\n",
    "                    tf.data.TFRecordDataset,\n",
    "                    sloppy = is_training,\n",
    "                    cycle_length = cycle_length,\n",
    "                )\n",
    "            )\n",
    "            d = d.shuffle(buffer_size = 100)\n",
    "        else:\n",
    "            d = tf.data.TFRecordDataset(input_files)\n",
    "            d = d.repeat()\n",
    "        d = d.apply(\n",
    "            tf.contrib.data.map_and_batch(\n",
    "                lambda record: _decode_record(record, name_to_features),\n",
    "                batch_size = batch_size,\n",
    "                num_parallel_batches = num_cpu_threads,\n",
    "                drop_remainder = True,\n",
    "            )\n",
    "        )\n",
    "        return d\n",
    "\n",
    "    return input_fn"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:\n",
      "The TensorFlow contrib module will not be included in TensorFlow 2.0.\n",
      "For more information, please see:\n",
      "  * https://github.com/tensorflow/community/blob/master/rfcs/20180907-contrib-sunset.md\n",
      "  * https://github.com/tensorflow/addons\n",
      "  * https://github.com/tensorflow/io (for I/O related ops)\n",
      "If you depend on functionality not listed there, please file an issue.\n",
      "\n",
      "WARNING:tensorflow:From <ipython-input-15-d06fc999c80c>:41: parallel_interleave (from tensorflow.contrib.data.python.ops.interleave_ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use `tf.data.experimental.parallel_interleave(...)`.\n",
      "WARNING:tensorflow:From /home/husein/.local/lib/python3.6/site-packages/tensorflow_core/contrib/data/python/ops/interleave_ops.py:77: parallel_interleave (from tensorflow.python.data.experimental.ops.interleave_ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use `tf.data.Dataset.interleave(map_func, cycle_length, block_length, num_parallel_calls=tf.data.experimental.AUTOTUNE)` instead. If sloppy execution is desired, use `tf.data.Options.experimental_determinstic`.\n",
      "WARNING:tensorflow:From <ipython-input-15-d06fc999c80c>:53: map_and_batch (from tensorflow.contrib.data.python.ops.batching) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use `tf.data.experimental.map_and_batch(...)`.\n",
      "WARNING:tensorflow:From /home/husein/.local/lib/python3.6/site-packages/tensorflow_core/contrib/data/python/ops/batching.py:276: map_and_batch (from tensorflow.python.data.experimental.ops.batching) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use `tf.data.Dataset.map(map_func, num_parallel_calls)` followed by `tf.data.Dataset.batch(batch_size, drop_remainder)`. Static tf.data optimizations will take care of using the fused implementation.\n",
      "WARNING:tensorflow:From /home/husein/.local/lib/python3.6/site-packages/tensorflow_core/python/autograph/converters/directives.py:119: The name tf.parse_single_example is deprecated. Please use tf.io.parse_single_example instead.\n",
      "\n",
      "WARNING:tensorflow:From <ipython-input-15-d06fc999c80c>:10: to_int32 (from tensorflow.python.ops.math_ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use `tf.cast` instead.\n"
     ]
    }
   ],
   "source": [
    "files = tf.io.gfile.glob('gs://mesolitica-tpu-general/pegasus-summarization-data/*.tfrecord')\n",
    "input_fn = input_fn_builder(files, 512, 512, 0, True)\n",
    "dataset = input_fn({'batch_size': 1})\n",
    "dataset = dataset._make_one_shot_iterator().get_next()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "sess = tf.Session()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "r = sess.run(dataset)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "import tokenization\n",
    "\n",
    "tokenizer = tokenization.FullTokenizer(\n",
    "    vocab_file='pegasus.wordpiece', do_lower_case=False\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['(',\n",
       " 'CN',\n",
       " '##N',\n",
       " ')',\n",
       " '-',\n",
       " 'Real',\n",
       " 'Madrid',\n",
       " 'telah',\n",
       " 'melakukan',\n",
       " 'sedikit',\n",
       " 'sebanyak',\n",
       " ',',\n",
       " 'jadi',\n",
       " 'sekarang',\n",
       " 'terser',\n",
       " '##ah',\n",
       " 'kepada',\n",
       " 'pesaing',\n",
       " 'hebat',\n",
       " 'Barcelona',\n",
       " 'untuk',\n",
       " 'memainkan',\n",
       " 'peranan',\n",
       " 'mereka',\n",
       " 'jika',\n",
       " 'final',\n",
       " 'Cop',\n",
       " '##a',\n",
       " 'del',\n",
       " 'Re',\n",
       " '##y',\n",
       " 'tahun',\n",
       " 'ini',\n",
       " 'akan',\n",
       " 'menjadi',\n",
       " 'lanjutan',\n",
       " 'dari',\n",
       " \"'\",\n",
       " 'El',\n",
       " 'Cla',\n",
       " '##si',\n",
       " '##co',\n",
       " \"'\",\n",
       " '.',\n",
       " 'Mend',\n",
       " '##ahului',\n",
       " '3',\n",
       " '-',\n",
       " '0',\n",
       " 'menentang',\n",
       " 'Atlet',\n",
       " '##ico',\n",
       " 'Madrid',\n",
       " ',',\n",
       " 'pesaing',\n",
       " 'utama',\n",
       " 'Real',\n",
       " 'yang',\n",
       " 'lain',\n",
       " ',',\n",
       " 'Cris',\n",
       " '##tian',\n",
       " '##o',\n",
       " 'Ronaldo',\n",
       " 'menjaringkan',\n",
       " 'dua',\n",
       " 'penalti',\n",
       " 'awal',\n",
       " 'untuk',\n",
       " 'merangkul',\n",
       " 'kemenangan',\n",
       " 'agregat',\n",
       " '5',\n",
       " '-',\n",
       " '0',\n",
       " 'yang',\n",
       " 'paling',\n",
       " 'selesa',\n",
       " 'di',\n",
       " 'separuh',\n",
       " 'akhir',\n",
       " '.',\n",
       " 'Peny',\n",
       " '##erang',\n",
       " 'Portugal',\n",
       " 'itu',\n",
       " 'yang',\n",
       " 'kini',\n",
       " 'menjalani',\n",
       " 'larangan',\n",
       " 'liga',\n",
       " 'tiga',\n",
       " 'perlawanan',\n",
       " 'selepas',\n",
       " 'kad',\n",
       " 'merah',\n",
       " 'kerana',\n",
       " 'melakukan',\n",
       " 'kekerasan',\n",
       " ',',\n",
       " 'dis',\n",
       " '##ambar',\n",
       " 'pem',\n",
       " '##etik',\n",
       " 'api',\n",
       " 'rokok',\n",
       " 'ketika',\n",
       " 'menuju',\n",
       " 'ke',\n",
       " 'terowong',\n",
       " 'pada',\n",
       " 'waktu',\n",
       " 'rehat',\n",
       " 'tetapi',\n",
       " 'muncul',\n",
       " 'semula',\n",
       " 'selepas',\n",
       " 'itu',\n",
       " 'tanpa',\n",
       " 'menunjukkan',\n",
       " 'kesan',\n",
       " 'buruk',\n",
       " '.',\n",
       " 'Cara',\n",
       " 'kemenangan',\n",
       " 'itu',\n",
       " 'adalah',\n",
       " 'modi',\n",
       " '##c',\n",
       " '##um',\n",
       " 'dendam',\n",
       " 'kepada',\n",
       " 'kemenangan',\n",
       " '2',\n",
       " '-',\n",
       " '1',\n",
       " 'Atlet',\n",
       " '##ico',\n",
       " 'ketika',\n",
       " 'menentang',\n",
       " 'Real',\n",
       " 'dalam',\n",
       " 'aksi',\n",
       " 'final',\n",
       " 'Cop',\n",
       " '##a',\n",
       " 'del',\n",
       " 'Re',\n",
       " '##y',\n",
       " 'tahun',\n",
       " 'lalu',\n",
       " '.',\n",
       " 'Barcelona',\n",
       " 'akan',\n",
       " 'bertemu',\n",
       " 'Real',\n",
       " 'dalam',\n",
       " 'penentu',\n",
       " 'April',\n",
       " 'ini',\n",
       " 'sekiranya',\n",
       " 'jurulatih',\n",
       " 'skuad',\n",
       " 'kendalian',\n",
       " 'Ger',\n",
       " '##ardo',\n",
       " 'Martin',\n",
       " '##o',\n",
       " 'itu',\n",
       " 'mempertahankan',\n",
       " 'kedudukan',\n",
       " '2',\n",
       " '-',\n",
       " '0',\n",
       " 'di',\n",
       " 'Real',\n",
       " 'Soci',\n",
       " '##eda',\n",
       " '##d',\n",
       " 'pada',\n",
       " 'aksi',\n",
       " 'separuh',\n",
       " 'akhir',\n",
       " 'kedua',\n",
       " 'Rabu',\n",
       " 'ini',\n",
       " '.',\n",
       " 'Need',\n",
       " '##ing',\n",
       " 'an',\n",
       " 'early',\n",
       " 'go',\n",
       " '##al',\n",
       " 'to',\n",
       " 'have',\n",
       " 'any',\n",
       " 'ho',\n",
       " '##pe',\n",
       " 'of',\n",
       " 'over',\n",
       " '##tur',\n",
       " '##ning',\n",
       " 'the',\n",
       " 'o',\n",
       " '##d',\n",
       " '##ds',\n",
       " ',',\n",
       " 'Atlet',\n",
       " '##ico',\n",
       " 'made',\n",
       " 'the',\n",
       " 'wor',\n",
       " '##st',\n",
       " 'possible',\n",
       " 'start',\n",
       " 'when',\n",
       " 'Ronaldo',\n",
       " 'was',\n",
       " 'trip',\n",
       " '##ped',\n",
       " 'in',\n",
       " 'the',\n",
       " 'box',\n",
       " 'after',\n",
       " 'six',\n",
       " 'minutes',\n",
       " '-',\n",
       " 'and',\n",
       " 'the',\n",
       " 're',\n",
       " '##ign',\n",
       " '##ing',\n",
       " 'FIFA',\n",
       " 'Ball',\n",
       " '##on',\n",
       " 'd',\n",
       " \"'\",\n",
       " 'Or',\n",
       " 'hold',\n",
       " '##er',\n",
       " 'made',\n",
       " 'no',\n",
       " 'mis',\n",
       " '##take',\n",
       " 'from',\n",
       " 'the',\n",
       " 'result',\n",
       " '##ing',\n",
       " 'spot',\n",
       " '-',\n",
       " 'ki',\n",
       " '##ck',\n",
       " '.',\n",
       " 'Sepuluh',\n",
       " 'minit',\n",
       " 'kemudian',\n",
       " ',',\n",
       " 'perlawanan',\n",
       " 'di',\n",
       " 'stadium',\n",
       " 'Vic',\n",
       " '##ente',\n",
       " 'Cal',\n",
       " '##der',\n",
       " '##on',\n",
       " 'itu',\n",
       " 'berjaya',\n",
       " 'ditamatkan',\n",
       " 'dengan',\n",
       " 'berkesan',\n",
       " 'apabila',\n",
       " 'Ronaldo',\n",
       " 'menjaringkan',\n",
       " 'penalti',\n",
       " 'keduanya',\n",
       " 'selepas',\n",
       " 'Gar',\n",
       " '##eth',\n",
       " 'Bal',\n",
       " '##e',\n",
       " 'dijatuhkan',\n",
       " '.',\n",
       " 'Walaupun',\n",
       " 'Atlet',\n",
       " '##ico',\n",
       " 'berdepan',\n",
       " 'dengan',\n",
       " 'aksi',\n",
       " 'cemerlang',\n",
       " 'musim',\n",
       " 'ini',\n",
       " ',',\n",
       " 'mereka',\n",
       " 'kekurangan',\n",
       " 'perc',\n",
       " '##ikan',\n",
       " 'api',\n",
       " 'apabila',\n",
       " 'Real',\n",
       " 'mel',\n",
       " '##edak',\n",
       " 'gol',\n",
       " 'ke',\n",
       " 'perlawanan',\n",
       " 'akhir',\n",
       " 'Cop',\n",
       " '##a',\n",
       " 'del',\n",
       " 'Re',\n",
       " '##y',\n",
       " 'ketiga',\n",
       " 'mereka',\n",
       " 'dalam',\n",
       " 'tempoh',\n",
       " 'empat',\n",
       " 'tahun',\n",
       " 'terakhir',\n",
       " '.',\n",
       " 'Fi',\n",
       " '##ore',\n",
       " '##n',\n",
       " '##tina',\n",
       " 'juga',\n",
       " 'berjaya',\n",
       " 'menembusi',\n",
       " 'perlawanan',\n",
       " 'akhir',\n",
       " 'piala',\n",
       " 'domestik',\n",
       " 'mereka',\n",
       " 'tetapi',\n",
       " 'kekalahan',\n",
       " 'agregat',\n",
       " '3',\n",
       " '-',\n",
       " '2',\n",
       " 'mereka',\n",
       " 'di',\n",
       " 'U',\n",
       " '##din',\n",
       " '##ese',\n",
       " ',',\n",
       " 'membi',\n",
       " '##da',\n",
       " 'untuk',\n",
       " 'mencapai',\n",
       " 'perlawanan',\n",
       " 'akhir',\n",
       " 'Piala',\n",
       " 'Itali',\n",
       " 'pertama',\n",
       " 'mereka',\n",
       " 'sejak',\n",
       " '192',\n",
       " '##2',\n",
       " ',',\n",
       " 'jauh',\n",
       " 'lebih',\n",
       " 'drama',\n",
       " '##tik',\n",
       " '.',\n",
       " 'Tr',\n",
       " '##ailing',\n",
       " '2',\n",
       " '-',\n",
       " '1',\n",
       " 'dari',\n",
       " 'leg',\n",
       " 'pertama',\n",
       " ',',\n",
       " 'Fi',\n",
       " '##ore',\n",
       " '##n',\n",
       " '##tina',\n",
       " 'menemukan',\n",
       " 'gol',\n",
       " 'pertama',\n",
       " 'dari',\n",
       " 'dua',\n",
       " 'gol',\n",
       " 'yang',\n",
       " 'mereka',\n",
       " 'butuh',\n",
       " '##kan',\n",
       " 'ketika',\n",
       " 'Manu',\n",
       " '##el',\n",
       " 'Pas',\n",
       " '##qu',\n",
       " '##al',\n",
       " 'melakukan',\n",
       " 'ter',\n",
       " '##ob',\n",
       " '##osan',\n",
       " 'setelah',\n",
       " 'hanya',\n",
       " '14',\n",
       " 'menit',\n",
       " '.',\n",
       " 'Tidak',\n",
       " 'lama',\n",
       " 'selepas',\n",
       " 'jam',\n",
       " 'itu',\n",
       " ',',\n",
       " 'Juan',\n",
       " 'Cu',\n",
       " '##ad',\n",
       " '##rad',\n",
       " '##o',\n",
       " 'dari',\n",
       " 'Colombia',\n",
       " 'menjaringkan',\n",
       " 'gol',\n",
       " 'yang',\n",
       " 'penting',\n",
       " 'dalam',\n",
       " 'permainan',\n",
       " 'yang',\n",
       " 'diperm',\n",
       " '##asalah',\n",
       " '##kan',\n",
       " 'oleh',\n",
       " 'pem',\n",
       " '##ecat',\n",
       " '##an',\n",
       " 'dua',\n",
       " 'pemain',\n",
       " 'U',\n",
       " '##din',\n",
       " '##ese',\n",
       " 'dalam',\n",
       " 'masa',\n",
       " 'pember',\n",
       " '##hen',\n",
       " '##tian',\n",
       " ',',\n",
       " 'dengan',\n",
       " 'pemain',\n",
       " 'Ghana',\n",
       " 'Em',\n",
       " '##man',\n",
       " '##uel',\n",
       " 'Ag',\n",
       " '##ye',\n",
       " '##man',\n",
       " '##g',\n",
       " 'Bad',\n",
       " '##u',\n",
       " 'dibuang',\n",
       " 'padang',\n",
       " 'dari',\n",
       " 'bangku',\n",
       " 'simpanan',\n",
       " '.',\n",
       " 'Ket',\n",
       " '##egangan',\n",
       " 'yang',\n",
       " 'tercetus',\n",
       " 'apabila',\n",
       " 'U',\n",
       " '##din',\n",
       " '##ese',\n",
       " 'secara',\n",
       " 'sempit',\n",
       " 'gagal',\n",
       " 'menjaringkan',\n",
       " 'gol',\n",
       " 'yang',\n",
       " 'akan',\n",
       " 'membawa',\n",
       " 'permainan',\n",
       " 'ke',\n",
       " 'masa',\n",
       " 'tambahan',\n",
       " '.',\n",
       " 'Fi',\n",
       " '##ore',\n",
       " '##n',\n",
       " '##tina',\n",
       " 'akan',\n",
       " 'bermain',\n",
       " 'pada',\n",
       " 'perlawanan',\n",
       " 'akhir',\n",
       " 'Mei',\n",
       " 'menentang',\n",
       " 'sama',\n",
       " 'ada',\n",
       " 'Nap',\n",
       " '##oli',\n",
       " 'atau',\n",
       " 'AS',\n",
       " 'Roma',\n",
       " ',',\n",
       " 'dengan',\n",
       " 'pasukan',\n",
       " 'dari',\n",
       " 'ibu',\n",
       " 'negara',\n",
       " 'mendahului',\n",
       " '3',\n",
       " '-',\n",
       " '2',\n",
       " 'selepas',\n",
       " 'perlawanan',\n",
       " 'pertama',\n",
       " 'di',\n",
       " 'tanah',\n",
       " 'air',\n",
       " '[PAD]',\n",
       " '[PAD]']"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tokenizer.convert_ids_to_tokens(r['input_ids'][0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['Real',\n",
       " 'Madrid',\n",
       " 'menewaskan',\n",
       " 'pesaing',\n",
       " 'bandar',\n",
       " ',',\n",
       " 'Atlet',\n",
       " '##ico',\n",
       " '5',\n",
       " '-',\n",
       " '0',\n",
       " 'menerusi',\n",
       " 'agregat',\n",
       " 'untuk',\n",
       " 'mara',\n",
       " 'ke',\n",
       " 'final',\n",
       " 'Cop',\n",
       " '##a',\n",
       " 'del',\n",
       " 'Re',\n",
       " '##y',\n",
       " '.',\n",
       " 'Barcelona',\n",
       " 'boleh',\n",
       " 'menyertai',\n",
       " 'mereka',\n",
       " 'jika',\n",
       " 'melindungi',\n",
       " 'kedudukan',\n",
       " '2',\n",
       " '-',\n",
       " '0',\n",
       " 'di',\n",
       " 'Real',\n",
       " 'Soci',\n",
       " '##eda',\n",
       " '##d',\n",
       " 'pada',\n",
       " 'Rabu',\n",
       " '.',\n",
       " 'Pemimpin',\n",
       " 'Liga',\n",
       " 'Perdana',\n",
       " 'Inggeris',\n",
       " ',',\n",
       " 'Chelsea',\n",
       " ',',\n",
       " 'menjatuhkan',\n",
       " 'mata',\n",
       " 'penting',\n",
       " 'di',\n",
       " 'West',\n",
       " 'Brom',\n",
       " '.',\n",
       " 'Fi',\n",
       " '##ore',\n",
       " '##n',\n",
       " '##tina',\n",
       " 'mencapai',\n",
       " 'perlawanan',\n",
       " 'akhir',\n",
       " 'Piala',\n",
       " 'Itali',\n",
       " 'dengan',\n",
       " 'kemenangan',\n",
       " 'kuku',\n",
       " 'ke',\n",
       " 'atas',\n",
       " 'U',\n",
       " '##din',\n",
       " '##ese',\n",
       " '.',\n",
       " '[CLS]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]',\n",
       " '[PAD]']"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tokenizer.convert_ids_to_tokens(r['target_ids'][0])"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
